package com.echo.spider;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.Authenticator;
import java.net.PasswordAuthentication;
import java.net.URL;

public class GetPage {
	
	public static String getContent(String urlStr){
		try{
			URL url = new URL(urlStr);
			BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()));
			String s = "";
			StringBuffer sb = new StringBuffer("");
			while((s=br.readLine())!=null){
				sb.append(s+"\r\n");
			}
			br.close();
			return sb.toString();
		}catch(Exception e){
			return "error open url: "+urlStr;
		}
	}
	
	//in case you may need to use proxy server
	public static void initProxy(String host, int port, final String username, final String password){
		Authenticator.setDefault(new Authenticator(){
			protected PasswordAuthentication getPasswordAuthentication(){
				return new PasswordAuthentication(username, new String(password).toCharArray());
			}
		});
		System.setProperty("http.proxyType", "4");
		System.setProperty("http.proxyPort", Integer.toString(port));
		System.setProperty("http.proxyHost", host);
		System.setProperty("http.proxySet", "true");
	}
	
	public static void main(String[] args)throws IOException{
		String url = "http://www.google.ca/webhp?hl=en";
		System.out.println(getContent(url));
	}

}
